/*
 *  This file is part of WWWPal.
 *
 *  Copyright (C) 2003  Dr. John Punin, Department of Computer Science
 *  Rensselaer Polytechnic Institute
 *
 *  WWWPal is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  WWWPal is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Foobar; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*
 *  This file is part of WWWPal.
 *
 *  Copyright (C) 2003  Dr. John Punin, Department of Computer Science
 *  Rensselaer Polytechnic Institute
 *
 *  WWWPal is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  WWWPal is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Foobar; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
#ifndef LOGR_WEBSTAT_H
#define LOGR_WEBSTAT_H

#include <sys/stat.h>
#include <sys/fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <ctype.h>
#include <sys/time.h>
#include <assert.h>
#include <gr_graph_fun.h>
#include <llist.h>
#include <alist.h>

#define LOGR_EXTENDED_LOG 0
#define LOGR_COMMON_LOG 1


#define SALTAVISTA "www.altavista.com"
#define SYAHOO "ink.yahoo.com"
#define SYAHOO2 "search.yahoo.com"
#define SGOOGLE "www.google.com"
#define SGOOGLE2 "google.netscape.com"
#define SINFOSEEK "infoseek.go.com"
#define SLYCOS "www.lycos.com"
#define SLYCOS2 "www.lycos.de"
#define SHOTBOT "www.hotbot.lycos.com"
#define SHOTBOT2 "click.hotbot.com"


#define S200 "200 - OK "
#define S206 "206 - Partial Content"
#define S301 "301 - Moved Permanently"
#define S302 "302 - Found"
#define S304 "304 - Not Modified"
#define S400 "400 - Bad Request"
#define S401 "401 - Unauthorized"
#define S402 "402 - Payment Required"
#define S403 "403 - Forbidden"
#define S404 "404 - Not Found"
#define S405 "405 - Not Allowed"
#define S408 "408 - Request Timeout"
#define S500 "500 - Internal Server Error"
#define S501 "501 - Method Not Implemented"


#define SMUNKNOWN "Unknown"
#define SMGET	"GET"
#define SMHEAD	"HEAD"
#define SMPOST  "POST"
#define SMPUT  "PUT"
#define SMOPTIONS  "OPTIONS"
#define SMDELETE  "DELETE"
#define SMTRACE  "TRACE"

#define SVUNKNOWN "Unknown"
#define SV10 "HTTP 1.0"
#define SV11 "HTTP 1.1"

#define TVUNKNOWN "unknown"
#define TV10 "HTTP/1.0"
#define TV11 "HTTP/1.1"

#define LOGR_TIME_FRAME  3600 /* seconds = 1 hour */

typedef enum _type_referer {
    RNOREFERER = 0x0, RSTATIC_REF = 0x1, RDYNAMIC_REF = 0x2 , RBOTH_REF = 0x3
} TypeReferer;

typedef enum _type_edge_referer {
    EXTERNAL_REFERER=1, INTERNAL_REFERER
} TypeEdgeReferer;

typedef enum _type_search_engine {
    TALTAVISTA=1, TYAHOO, TYAHOO2, TGOOGLE, TGOOGLE2, TINFOSEEK, TLYCOS, TLYCOS2, THOTBOT,  THOTBOT2
} SearchEngineType;

typedef enum _type_http_method {
    HTTP_METHOD_UNKNOWN = 0, HTTP_METHOD_GET = 1, HTTP_METHOD_HEAD, HTTP_METHOD_POST, HTTP_METHOD_PUT, HTTP_METHOD_OPTIONS, HTTP_METHOD_DELETE, HTTP_METHOD_TRACE
} HttpMethodType;

typedef enum _type_http_version {
    HTTP_VERSION_UNKNOWN = 0, HTTP_VERSION_10 = 1, HTTP_VERSION_11
} HttpVersionType;

typedef struct _web_stat_frame {
    char *logs_dir;
    char *base_webhost;
    char *local_domain;
    char *url_prefix;
    int logs_file_type;
    LNode_type *lcidr;
    short compute_user_sessions;
    short resolve_host_ip;
    int resolve_timeout;
    int maxidnum;
    long count_hits;
    long size_bytes;
    long nsearch;
    void *hours[24];
    void *days[31];
    void *months[12];
    LNode_type *start_dates;
    LNode_type *end_dates;
    LNode_type *lsites;
    Hashtable_type tsites;
    LNode_type *ldomains;
    LNode_type *luas;
    Hashtable_type tuas;
    LNode_type *lbrowsers;
    LNode_type *lspiders;
    LNode_type *lplatforms;
    LNode_type *alcodes;
    LNode_type *almethods;
    LNode_type *alversions;
    LNode_type *simple_pages;
    LNode_type *independent_pages;
    Hashtable_type tindependent_pages;
    LNode_type *simple_links;
    LNode_type *independent_links;
    Hashtable_type tindependent_links;
    LNode_type *entry_pages;
    Hashtable_type tentry_pages;
    LNode_type *lsites_ref;
    Hashtable_type tsites_ref;
    LNode_type *lurl_ext_ref;
    Hashtable_type turl_ext_ref;
    LNode_type *lurl_int_ref;
    Hashtable_type turl_int_ref;
    LNode_type *lkeywords;
    Hashtable_type keywords;
    Hashtable_type labels;
    LNode_type *lusers;	
    LNode_type *lgraphs;
    LNode_type *lpaths;
    LNode_type *ldirectories;
    Hashtable_type tdirectories;
    void *lcusers;
} WebStatStruct;

typedef struct _web_stat_frame *WebStatPtr;

WebStatPtr
new_web_stat(void);

void
delete_web_stat(WebStatPtr wsf);

void
clear_web_stat(WebStatPtr wsf);

void
reset_web_stat(WebStatPtr wsf);

void
LOGR_set_max_id_number(WebStatPtr wsf,int maxidnum);

int
LOGR_get_max_id_number(WebStatPtr wsf);

void
LOGR_set_logs_directory(WebStatPtr wsf,char *ldir);

const char *
LOGR_get_logs_directory(WebStatPtr wsf);

void
LOGR_set_local_domain(WebStatPtr wsf,char *ldom);

const char *
LOGR_get_local_domain(WebStatPtr wsf);

void
LOGR_set_base_webhost(WebStatPtr wsf,char *bwh);

const char *
LOGR_get_base_webhost(WebStatPtr wsf);

const char *
LOGR_get_url_prefix(WebStatPtr wsf);

void
LOGR_set_url_prefix(WebStatPtr wsf,char *ulp);

void
LOGR_set_logs_filetype(WebStatPtr wsf,int lft);

int
LOGR_get_logs_filetype(WebStatPtr wsf);

void
LOGR_set_compute_user_sessions(WebStatPtr wsf);
void
LOGR_reset_compute_user_sessions(WebStatPtr wsf);

short
LOGR_get_compute_user_sessions(WebStatPtr wsf);

void
LOGR_set_resolve_timeout(WebStatPtr wsf,int resolve_timeout);

int
LOGR_get_resolve_timeout(WebStatPtr wsf);

void
LOGR_set_resolve_host_ip(WebStatPtr wsf);

void
LOGR_reset_resolve_host_ip(WebStatPtr wsf);

short
LOGR_get_resolve_host_ip(WebStatPtr wsf);

void
LOGR_set_start_date(WebStatPtr wsf,time_t t);

LNode_type *
LOGR_get_start_dates(WebStatPtr wsf);

void
LOGR_set_end_date(WebStatPtr wsf,time_t t);


LNode_type *
LOGR_get_end_dates(WebStatPtr wsf);

time_t
LOGR_get_first_start_date(WebStatPtr wsf);

time_t
LOGR_get_first_end_date(WebStatPtr wsf);

time_t
LOGR_get_minimum_start_date(WebStatPtr wsf);

time_t
LOGR_get_maximum_end_date(WebStatPtr wsf);

void
LOGR_clear_list_cidr(WebStatPtr wsf);

void
LOGR_add_cidr(WebStatPtr wsf, char *cidr);

short
LOGR_is_ip_local(WebStatPtr wsf, char *ip);

void
LOGR_add_domain(WebStatPtr wsf, char *domain,long nacc,long size,long npages);

void
LOGR_add_site(WebStatPtr wsf, char *site,int cur_size,int ishtmlpage);

void
LOGR_add_size_site(WebStatPtr wsf, char *site, char *size);

void
LOGR_add_num_size_site(WebStatPtr wsf, char *site, int size);

LNode_type *
LOGR_get_list_sites(WebStatPtr wsf);

LNode_type *
LOGR_get_list_domains(WebStatPtr wsf);

void
LOGR_add_site_ref(WebStatPtr wsf, char *site);

void
LOGR_add_site_ref2(WebStatPtr wsf, char *site,int size);

void
LOGR_add_site_ref3(WebStatPtr wsf, char *site,int size,TypeReferer tr);

void
LOGR_add_size_site_ref(WebStatPtr wsf, char *site, char *size);

LNode_type *
LOGR_get_list_sites_ref(WebStatPtr wsf);

LNode_type *
LOGR_get_list_directories(WebStatPtr wsf);

void
LOGR_add_browser(WebStatPtr wsf, char *browser, int nsize,int ishtmlpage);

void
LOGR_add_size_browser(WebStatPtr wsf, char *browser, char *size);

void
LOGR_add_size_browser_num(WebStatPtr wsf, char *browser, int nsize);

LNode_type *
LOGR_get_list_ua(WebStatPtr wsf);

LNode_type *
LOGR_get_list_browser(WebStatPtr wsf);

LNode_type *
LOGR_get_list_platform(WebStatPtr wsf);

LNode_type *
LOGR_get_list_spider(WebStatPtr wsf);

double
LOGR_get_total_browser_size(WebStatPtr wsf);

long
LOGR_get_total_browser_naccesses(WebStatPtr wsf);


void
LOGR_process_browsers_type_version_platform(WebStatPtr wsf);

void
LOGR_process_simple_pages(WebStatPtr wsf,GraphFrame *gf,short verify);

void
LOGR_process_simple_links(WebStatPtr wsf,GraphFrame *gf,short verify);

void
LOGR_process_directories(WebStatPtr wsf,GraphFrame *gf);

void
LOGR_process_sites(WebStatPtr wsf,int what);

void
LOGR_process_domains(WebStatPtr wsf);

void
LOGR_add_independent_page(WebStatPtr wsf, char *url,long size, int http_code,
		     char *mime);

LNode_type *
LOGR_get_list_simple_pages(WebStatPtr wsf);

LNode_type *
LOGR_get_list_simple_links(WebStatPtr wsf);

LNode_type *
LOGR_get_list_independent_pages(WebStatPtr wsf);

void
LOGR_add_url_external_referer(WebStatPtr wsf, char *url,char *turl,int csize,
			      int http_code);

LNode_type *
LOGR_get_list_url_external_referers(WebStatPtr wsf);

void
LOGR_add_url_internal_referer(WebStatPtr wsf, char *url,GraphFrame *gf,
			      int csize);

LNode_type *
LOGR_get_list_url_internal_referers(WebStatPtr wsf);

void
LOGR_add_entry_page(WebStatPtr wsf, char *url,GraphFrame *gf,TypeReferer tr);

LNode_type *
LOGR_get_alist_codes(WebStatPtr wsf);

void
LOGR_add_http_code(WebStatPtr wsf,int code,int cur_size, int ishtmlpage);

LNode_type *
LOGR_get_alist_methods(WebStatPtr wsf);

void
LOGR_add_http_method(WebStatPtr wsf,HttpMethodType method,int cur_size, 
		     int ishtmlpage);

LNode_type *
LOGR_get_alist_versions(WebStatPtr wsf);

void
LOGR_add_http_version(WebStatPtr wsf,HttpVersionType version,int cur_size, 
		      int ishtmlpage);


LNode_type *
LOGR_get_list_entry_pages(WebStatPtr wsf);

double
LOGR_get_total_site_size(WebStatPtr wsf);

long
LOGR_get_total_site_naccesses(WebStatPtr wsf);

Hashtable_type *
LOGR_get_table_keywords(WebStatPtr wsf);

long
LOGR_get_total_number_searches(WebStatPtr wsf);

void
LOGR_add_date(WebStatPtr wsf, char *sdate, int ishtmlpage, int sbytes);

void *
LOGR_get_info_hours(WebStatPtr wsf,int i);

void *
LOGR_get_info_days(WebStatPtr wsf,int i);

void *
LOGR_get_info_months(WebStatPtr wsf,int i);


extern char *
LOGR_string_month(int month);

extern char *
LOGR_get_xml_string(char *str);

LNode_type *
LOGR_get_list_independent_links(WebStatPtr wsf);

void
LOGR_add_independent_link(WebStatPtr wsf, char *urls,char *urlt,GraphFrame *gf,
		     char *ua);

void
LOGR_add_directory(WebStatPtr wsf, char *url, int hits,int bytes);

short
LOGR_is_user_agent_spider(WebStatPtr wsf,char *ua_name);

LNode_type *
LOGR_get_list_users(WebStatPtr wsf);

LNode_type *
LOGR_get_list_paths(WebStatPtr wsf);

LNode_type *
LOGR_get_list_graphs(WebStatPtr wsf);

void
LOGR_get_total_number_pages(WebStatPtr wsf,GraphFrame *gf,int *nhtml,
			    int *nnhtml, int *nobj);
void
LOGR_get_total_number_hyperlinks(WebStatPtr wsf,GraphFrame *gf,int *hhtml,
				 int *hnhtml);

void
LOGR_get_total_number_entry_pages(WebStatPtr wsf,GraphFrame *gf,int *ehtml,
				  int *enhtml);

void
LOGR_get_total_number_referers(WebStatPtr wsf,GraphFrame *gf,int *rhost,
			       int *rsa);

void
LOGR_get_average_by_time(WebStatPtr wsf,double *ave_req_day,
			 double *ave_req_hour, double *ave_size_day, 
			 double *ave_size_hour);

long
LOGR_get_entry_page_hits(WebStatPtr wsf,char *url);

extern const char *
LOGR_get_string_code(int code);

extern const char *
LOGR_get_string_method(int method);

extern const char *
LOGR_get_string_version(int version);

extern const char *
LOGR_get_token_version(int version);

extern char *
LOGR_get_xml_escape_string(char *str);

extern char *
LOGR_get_xml_escape_nonascii_string(char *str);

extern char *
LOGR_get_xml_nescape_string(char *str);

extern char *
LOGR_IPfromhost( char *hostname);

extern char *
LOGR_hostfromIP( char *ip ,int timeout);

extern short
LOGR_is_ip(char *str);

extern char *
LOGR_get_long_country_code(char *code);

extern short
LOGR_is_address_local(WebStatPtr wsf,char *address);

extern short
LOGR_is_address_in_subnet( char *addr, char *subnet, int plen);
    
extern void
LOGR_to_lower(char *str);

extern void
delete_list_strings(LNode_type *llist);

extern char *
HTStringTimeApache (time_t t);

extern void
HTBufferTimeApache (time_t t, char *buffer, int n);


extern HttpVersionType
LOGR_get_http_version_type(char *version);

extern HttpMethodType
LOGR_get_http_method_type(char *method);

extern char *
LOGR_get_absolute_uri(WebStatPtr wsf, GraphFrame *gf,char *uri,const char *base_webhost,int *new);

#endif



